"""58同城租房信息爬取"""
# -*- coding: utf-8 -*-
# @Author: yqbao
# @GiteeURL: https://gitee.com/yqbao
# @Date: 2019/8/25 15:01
# @Version: v.0.0

import requests
import json
import re
import time
import base64
from io import BytesIO
from fontTools.ttLib import TTFont


def query(url):
    """获取数据"""
    headers = {
        'cookie': 'f=n; commontopbar_new_city_info=9811%7C%E5%B7%B4%E4%B8%AD%7Cbazhong; f=n; commontopbar_new_city_info=9811%7C%E5%B7%B4%E4%B8%AD%7Cbazhong; userid360_xml=C1A19DF67D717E687F1AABC2187085E0; time_create=1569298680617; 58home=bazhong; f=n; id58=e87rZl1iC/A6g3qkEA2pAg==; city=bazhong; commontopbar_new_city_info=9811%7C%E5%B7%B4%E4%B8%AD%7Cbazhong; commontopbar_ipcity=bazhong%7C%E5%B7%B4%E4%B8%AD%7C0; 58tj_uuid=764f0cc8-d8fa-4b66-b940-aa930e5bac96; als=0; xxzl_deviceid=VJWRrvY2r3HrwQs8uxc0koKt5DbYHlcBXNl50Gw%2B94XCjckzqrLZqwftuvdaa%2FzT; new_uv=2; utm_source=; spm=; init_refer=; new_session=0',
        'upgrade-insecure-requests': '1',
        'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.131 Safari/537.36',
    }
    try:
        req = requests.get(url=url, headers=headers)  # 封装请求头并请求网页
        if req.status_code == 200:  # 成功，则获取响应的html内容，并返回
            response = req.content.decode('utf-8')
            return response
        else:
            return None
    except requests.exceptions.RequestException:
        return None


def decodes(encode_data, base64_str):
    """字符解码"""
    text = list(encode_data)
    font = TTFont(BytesIO(base64.decodebytes(base64_str.encode())))  # 将编码字符串base64_str解码，并生成字体文件
    decode_dict = font['cmap'].tables[0].cmap  # 找出基础字形名称的列表
    for j in range(len(encode_data)):
        try:
            code = int(text[j].encode('unicode-escape').decode()[-4:], 16)  # 将爬取到的信息一次编码，然后切片取指定值
            if code in decode_dict.keys():
                text[j] = str(int(decode_dict[code][-2:]) - 1)  # 根据code键找出decode_dict字典中对应的值，并-1
        except ValueError:
            j += 1
            continue
    return ''.join(text)


def analysis(content):
    """解析数据"""
    # 实测发现该编码串会在一定时间后改变，导致解密出的数字混乱
    # 于是此处正则解析该编码串
    global bse64_str
    try:
        bse64_str = str(re.findall("charset=utf-8;base64,(.*?)'\)", content)[0])  # 解析base64型编码字符串
    except IndexError as e:
        print(e.args)

    # ########################### 使用PyQuery 解析数据 ##################################
    from pyquery import PyQuery as Pq
    doc = Pq(content)  # 初始化PyQuery
    items = doc('li.house-cell')  # 获取<li class="house-cell" ....></li>租房信息列表
    for item in items.items():  # 在信息列表中解析需要的数据
        yield {
            'title': decodes(item.find('h2 a.strongbox').text(), bse64_str),  # 解码加密的字符数据
            'image': item.find('div.img-list a img').attr('lazy_src'),
            'rooms': decodes(item.find('div.des p.room').text(), bse64_str),
            'money': decodes(item.find('div.money').text(), bse64_str),
        }

    """
    # ########################### 使用 BeautifulSoup 解析数据 ##################################
    from bs4 import BeautifulSoup
    soup = BeautifulSoup(content, 'lxml')  # 初始化BeautifulSoup
    items = soup.find_all(name='li', attrs={'class': 'house-cell'})  # 获取<li class="house-cell" ....></li>租房信息列表
    for item in items:  # 在信息列表中解析需要的数据
        yield {
            'title': decodes(item.find(name='a', attrs={'class': 'strongbox'}).string, bse64_str),  # 解码加密的字符数据
            'image': item.find(name='img').attrs['lazy_src'],
            'rooms': decodes(item.find(name='p', attrs={'class': 'room'}).string, bse64_str),
            'money': decodes(item.find(name='div', attrs={'class': 'money'}).get_text(), bse64_str),
        }
    """
    """
    # ########################### 使用 Xpath 解析数据 ##################################
    from lxml import etree
    html = etree.HTML(content)  # 初始化Xpath
    items = html.xpath('//li[@class="house-cell"]')  # 获取<li class="house-cell" ....></li>租房信息列表
    for item in items:  # 在信息列表中解析需要的数据
        yield {
            'title': decodes(item.xpath('.//a[@class="strongbox"]/text()')[0], bse64_str),  # 解码加密的字符数据
            'image': item.xpath('.//img/@lazy_src')[0],
            'rooms': decodes(item.xpath('.//p[@class="room"]/text()')[0], bse64_str),
            'money': decodes(item.xpath('.//div[@class="money"]/b[@class="strongbox"]/text()')[0] +
                             item.xpath('.//div[@class="money"]/text()')[1], bse64_str),
        }
    """


def save(content):
    """保存数据"""
    # 字典数据转换为csv保存写入
    with open("result.txt", 'a', encoding="utf-8") as f:
        f.write(json.dumps(content, ensure_ascii=False) + "\n")


def main(page):
    """调度"""
    url = 'https://bazhong.58.com/chuzu/pn{}/?ClickID=1'.format(page)
    html = query(url)
    if html:
        data = analysis(html)
        for item in data:
            print(item)
            save(item)


if __name__ == '__main__':
    pages = input('获取指定页,请输入指定页页码（1-70），否则按 "a" 获取全部页：')
    if pages == 'a':
        for i in range(1, 71):
            print('#' * 50 + ' 正在爬取第 {} 页 '.format(i) + '#' * 50)
            main(str(i))
            time.sleep(25)
    else:
        main(pages)
